package cn.edu360t.etl

import cn.edu360.DataUtils
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Row, SQLContext}
import org.apache.spark.sql.types._
import org.apache.spark.{SparkConf, SparkContext}

//将Bz2格式文件转换为Parquet形式(离线数据处理)
object Bz2Parquet {
  def main(args: Array[String]): Unit = {
    //使用args进行参数判断
    if (args.length != 2) {
      println(
        """
          |cn.edu360teacher.etl.Bz2Parquet
          |请输入2个正确的参数:inputpath outpath
        """.stripMargin
      )
      sys.exit(1)
    }
    //接收参数
    val Array(inputpath, outpath) = args
    //先把模板代码写上
    val conf: SparkConf = new SparkConf()
      //本地运行设置
      .setMaster("local[*]")
      .setAppName("日志格式转换为Parquet文件")
      //设置序列化
      .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      //设置压缩方式
      .set("spark.sql.parquet.compression", "codec	snappy")
    //SparkContext的对象 用于读取静态文件
    val sc: SparkContext = new SparkContext(conf)
    //创建 SQLcontext
    val sqlcontext: SQLContext = new SQLContext(sc)
    //导入隐式转换
    import sqlcontext.implicits._
    //业务逻辑
    //1:读取数据
    val yuanData: RDD[String] = sc.textFile(inputpath)
    // 2:整理数据 => DataFrame = RDD[Row] + StructType
    // 创建StructType
    val schema = StructType(List(
      StructField("sessionid", StringType),
      StructField("advertisersid", IntegerType),
      StructField("adorderid", IntegerType),
      StructField("adcreativeid", IntegerType),
      StructField("adplatformproviderid", IntegerType),
      StructField("sdkversion", StringType),
      StructField("adplatformkey", StringType),
      StructField("putinmodeltype", IntegerType),
      StructField("requestmode", IntegerType),
      StructField("adprice", DoubleType),
      StructField("adppprice", DoubleType),
      StructField("requestdate", StringType),
      StructField("ip", StringType),
      StructField("appid", StringType),
      StructField("appname", StringType),
      StructField("uuid", StringType),
      StructField("device", StringType),
      StructField("client", IntegerType),
      StructField("osversion", StringType),
      StructField("density", StringType),
      StructField("pw", IntegerType),
      StructField("ph", IntegerType),
      StructField("long", StringType),
      StructField("lat", StringType),
      StructField("provincename", StringType),
      StructField("cityname", StringType),
      StructField("ispid", IntegerType),
      StructField("ispname", StringType),
      StructField("networkmannerid", IntegerType),
      StructField("networkmannername", StringType),
      StructField("iseffective", IntegerType),
      StructField("isbilling", IntegerType),
      StructField("adspacetype", IntegerType),
      StructField("adspacetypename", StringType),
      StructField("devicetype", IntegerType),
      StructField("processnode", IntegerType),
      StructField("apptype", IntegerType),
      StructField("district", StringType),
      StructField("paymode", IntegerType),
      StructField("isbid", IntegerType),
      StructField("bidprice", DoubleType),
      StructField("winprice", DoubleType),
      StructField("iswin", IntegerType),
      StructField("cur", StringType),
      StructField("rate", DoubleType),
      StructField("cnywinprice", DoubleType),
      StructField("imei", StringType),
      StructField("mac", StringType),
      StructField("idfa", StringType),
      StructField("openudid", StringType),
      StructField("androidid", StringType),
      StructField("rtbprovince", StringType),
      StructField("rtbcity", StringType),
      StructField("rtbdistrict", StringType),
      StructField("rtbstreet", StringType),
      StructField("storeurl", StringType),
      StructField("realip", StringType),
      StructField("isqualityapp", IntegerType),
      StructField("bidfloor", DoubleType),
      StructField("aw", IntegerType),
      StructField("ah", IntegerType),
      StructField("imeimd5", StringType),
      StructField("macmd5", StringType),
      StructField("idfamd5", StringType),
      StructField("openudidmd5", StringType),
      StructField("androididmd5", StringType),
      StructField("imeisha1", StringType),
      StructField("macsha1", StringType),
      StructField("idfasha1", StringType),
      StructField("openudidsha1", StringType),
      StructField("androididsha1", StringType),
      StructField("uuidunknow", StringType),
      StructField("userid", StringType),
      StructField("iptype", IntegerType),
      StructField("initbidprice", DoubleType),
      StructField("adpayment", DoubleType),
      StructField("agentrate", DoubleType),
      StructField("lrate", DoubleType),
      StructField("adxrate", DoubleType),
      StructField("title", StringType),
      StructField("keywords", StringType),
      StructField("tagid", StringType),
      StructField("callbackdate", StringType),
      StructField("channelid", StringType),
      StructField("mediatype", IntegerType)
    ))
    val filtered: RDD[Array[String]] = yuanData.map(line=>line.split(",",-1)).filter(line=>line.length>=85)
    import cn.edu360t.beans.RichString._
    val maped: RDD[Row] = filtered.map(arr => Row(
      arr(0),
      DataUtils.parseInt(arr(1)),
      arr(2).parseInt,
      arr(3).parseInt,
      arr(4).parseInt,
      arr(5),
      arr(6),
      arr(7).parseInt,
      arr(8).parseInt,
      arr(9).parseDouble,
      arr(10).parseDouble,
      arr(11),
      arr(12),
      arr(13),
      arr(14),
      arr(15),
      arr(16),
      arr(17).parseInt,
      arr(18),
      arr(19),
      arr(20).parseInt,
      arr(21).parseInt,
      arr(22),
      arr(23),
      arr(24),
      arr(25),
      arr(26).parseInt,
      arr(27),
      arr(28).parseInt,
      arr(29),
      arr(30).parseInt,
      arr(31).parseInt,
      arr(32).parseInt,
      arr(33),
      arr(34).parseInt,
      arr(35).parseInt,
      arr(36).parseInt,
      arr(37),
      arr(38).parseInt,
      arr(39).parseInt,
      arr(40).parseDouble,
      arr(41).parseDouble,
      arr(42).parseInt,
      arr(43),
      arr(44).parseDouble,
      arr(45).parseDouble,
      arr(46),
      arr(47),
      arr(48),
      arr(49),
      arr(50),
      arr(51),
      arr(52),
      arr(53),
      arr(54),
      arr(55),
      arr(56),
      arr(57).parseInt,
      arr(58).parseDouble,
      arr(59).parseInt,
      arr(60).parseInt,
      arr(61),
      arr(62),
      arr(63),
      arr(64),
      arr(65),
      arr(66),
      arr(67),
      arr(68),
      arr(69),
      arr(70),
      arr(71),
      arr(72),
      arr(73).parseInt,
      arr(74).parseDouble,
      arr(75).parseDouble,
      arr(76).parseDouble,
      arr(77).parseDouble,
      arr(78).parseDouble,
      arr(79),
      arr(80),
      arr(81),
      arr(82),
      arr(83),
      arr(84).parseInt
    ))
    val dataFrame = sqlcontext.createDataFrame(maped, schema)
    // 数据写出
    dataFrame.write.parquet(outpath)
    //3：将数据转换为Parquet
    //4：读取Parquet文件
    sc.stop()
  }
}
